import pandas as pd
import numpy as np
import plotly.express as px
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import balanced_accuracy_score, accuracy_score, f1_score
from sklearn.dummy import DummyClassifier
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from sklearn.metrics import make_scorer
import joblib
from sklearn.inspection import permutation_importance
from sklearn.pipeline import Pipeline
from time import time
import plotly.io as pio
pio.renderers.default = "notebook"

# Getting warnings from seaborn, which are excluded here
import warnings
warnings.filterwarnings("ignore", category=FutureWarning)

def read_beans():  
    # Read the data, i should change the name of the columns, so units are contained  
    df = pd.read_excel('data/Dry_Bean_Dataset.xlsx', sheet_name='Dry_Beans_Dataset')
    return df

df = read_beans()
df.head()

print(f'The dataset has {df.shape[0]} rows and {df.shape[1]} columns')

The dataset has 13611 rows and 17 columns

print(f'The dataset has {df.isnull().sum().sum()} missing values.')

The dataset has 0 missing values.

class_distribution = df.groupby('Class').count()
fig = px.bar(class_distribution, x=class_distribution.index, y='Area', title='Class distribution')
fig.update_xaxes(categoryorder='total descending')
fig.show()

display(df.drop(columns='Class').aggregate(['min', 'max', 'mean', 'std']))

# check for duplicates
duplicates = df.duplicated().sum()
print(f'There are {duplicates} duplicates in the dataset.')

# what is the calss distribution of the duplicates
duplicates = df[df.duplicated()]
class_distribution = duplicates.groupby('Class').count()['Area'].to_frame().rename(columns={'Area': 'Duplicate Rows'})
display(class_distribution)

There are 68 duplicates in the dataset.

# drop duplicates
df = df.drop_duplicates()
print(f'The dataset has {df.shape[0]} rows and {df.shape[1]} columns')

The dataset has 13543 rows and 17 columns

# pairplot with seaborn
not_derived = ['Area', 'Perimeter', 'MajorAxisLength', 'MinorAxisLength', 'Eccentricity', 'ConvexArea', 'Class']
sns.pairplot(df.loc[:, not_derived], hue='Class', plot_kws={'alpha':0.1})

<seaborn.axisgrid.PairGrid at 0x74d8ce549e50>

# create a correlation matrix
corr = df.drop(columns='Class').corr().round(2)

# Define a custom colorscale
colorscale = [
    [0.0, "blue"],      # Blue for -1
    [0.5, "white"],     # White for 0
    [1.0, "red"]        # Red for 1
]

# Create the heatmap with the custom colorscale
fig = px.imshow(
    corr, 
    labels=dict(color='Correlation'), 
    x=corr.columns, 
    y=corr.columns, 
    text_auto=True, 
    color_continuous_scale=colorscale
)

fig.update_layout(title='Correlation Matrix', height=1000, width=1000)
fig.show()

not_derived = ['Area', 'Perimeter', 'MajorAxisLength', 'MinorAxisLength', 'Eccentricity', 'ConvexArea', 'Class']
df_melted = df[not_derived].melt(id_vars='Class', var_name='Feature', value_name='Value')

# Create a FacetGrid for each feature colored by class
g = sns.FacetGrid(df_melted, col='Feature', hue='Class', col_wrap=3, sharex=False, sharey=False, aspect=1.5, height=5)

# Map the histogram to each subplot
g.map(sns.histplot, 'Value', bins=50, kde=True, alpha=0.5, edgecolor='white')

# Add legends and adjust layout
g.add_legend(title='Class', title_fontsize='15', fontsize='12')
g.set_titles('{col_name}', size=16)
g.set_axis_labels('Value', 'Frequency', fontsize=14)

# Customize tick labels
for ax in g.axes.flatten():
    ax.tick_params(labelsize=12)  # Customize tick label size

# Add a main title
plt.subplots_adjust(top=0.92)
g.fig.suptitle('Distribution of not Derived Features by Class', fontsize=20)

# Show the plot
plt.show()

# Plotting boxplots for each feature grouped by class
fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(15, 15))
fig.suptitle('Boxplots of features by class', fontsize=20)

# Flatten the axes array for easy iteration
axes = axes.flatten()

for i, feature in enumerate(not_derived[:-1]):
    sns.boxplot(data=df, x='Class', y=feature, ax=axes[i])
    axes[i].set_title(feature)
    axes[i].tick_params(axis='x', rotation=45)  # Rotate x-axis tick labels
# Remove any extra subplots
for j in range(i+1, len(axes)):
    fig.delaxes(axes[j])

plt.tight_layout(rect=[0, 0, 1, 0.95])
plt.show()

not_derived = ['Area', 'Perimeter', 'MajorAxisLength', 'MinorAxisLength', 'Eccentricity', 'ConvexArea']
df_melted = df.drop(columns=not_derived).melt(id_vars='Class', var_name='Feature', value_name='Value')

# Create a FacetGrid for each feature colored by class
g = sns.FacetGrid(df_melted, col='Feature', hue='Class', col_wrap=3, sharex=False, sharey=False, aspect=1.5, height=3)

# Map the histogram to each subplot
g.map(sns.histplot, 'Value', bins=50, kde=True, alpha=0.5, edgecolor='white')

# Add legends and adjust layout
g.add_legend(title='Class', title_fontsize='15', fontsize='12')
g.set_titles('{col_name}', size=16)
g.set_axis_labels('Value', 'Frequency', fontsize=14)

# Customize tick labels
for ax in g.axes.flatten():
    ax.tick_params(labelsize=12)  # Customize tick label size

# Rotate x-tick labels for 'ShapeFactor2' by 45 degrees
for ax, title in zip(g.axes.flatten(), g.col_names):
    if title == 'ShapeFactor2':
        for label in ax.get_xticklabels():
            label.set_rotation(45)
            label.set_horizontalalignment('right')

# Add a main title
plt.subplots_adjust(top=0.92)
g.fig.suptitle('Distribution of feature engineered Features by Class', fontsize=20)

# Show the plot
plt.show()

# Plotting boxplots for each feature grouped by class
fig, axes = plt.subplots(nrows=4, ncols=3, figsize=(15, 15))
fig.suptitle('Boxplots of features by class', fontsize=20)

# Flatten the axes array for easy iteration
axes = axes.flatten()
derived = ['MinorAxisLength', 'AspectRation', 'EquivDiameter', 'Extent',
       'Solidity', 'roundness', 'Compactness', 'ShapeFactor1', 'ShapeFactor2',
       'ShapeFactor3', 'ShapeFactor4', 'Class']
for i, feature in enumerate(derived[:-1]):
    sns.boxplot(data=df, x='Class', y=feature, ax=axes[i])
    axes[i].set_title(feature)
    axes[i].tick_params(axis='x', rotation=45)  # Rotate x-axis tick labels
# Remove any extra subplots
for j in range(i+1, len(axes)):
    fig.delaxes(axes[j])

plt.tight_layout(rect=[0, 0, 1, 0.95])
plt.show()

# At firs I want to create dataframe with the results
df_results = pd.DataFrame(columns=['Model', 'Mean Balanced Accuracy Test', 'Std Balanced Accuracy Test', 
                                   'Mean Accuracy Test', 'Std Accuracy Test', 'Mean F1 Score Test', 
                                   'Std F1 Score Test', 'Mean Balanced Accuracy Train', 'Std Balanced Accuracy Train', 
                                   'Mean Accuracy Train', 'Std Accuracy Train', 'Mean F1 Score Train', 'Std F1 Score Train', 
                                   'Fit Time','Prediction Time'])

# scorer for the used metrics, this scorer will be use through the notebook
def scorer(y_true, y_pred):
    b_acc = balanced_accuracy_score(y_true, y_pred)
    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='weighted')
    return b_acc, acc, f1

# Split the data into train and test
X = df.drop(columns='Class')
y = df['Class']

def inter_dummy(df, df_results=df_results):
    # Split the data into train and test
    X = df.drop(columns='Class')
    y = df['Class']
    # fot the results
    b_accs_test = []
    accs_test = []
    f1s_test = []
    b_accs_train = []
    accs_train = []
    f1s_train = []
    fitting_times = []
    prediction_times = []
    for index in range(10):
        # train test split
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=index)

        # Create a dummy classifier
        dummy = DummyClassifier(strategy='most_frequent')
        a = time()
        dummy.fit(X_train, y_train)
        fitting_times.append(time()-a)
        b = time()
        y_pred = dummy.predict(X_test)
        prediction_times.append(time()-b)
        # Calculate the balanced accuracy
        b_acc_test, acc_test, f1_test = scorer(y_test, y_pred)
        b_acc_train, acc_train, f1_train = scorer(y_train, dummy.predict(X_train))

        # append the results
        b_accs_test.append(b_acc_test); accs_test.append(acc_test); f1s_test.append(f1_test)
        b_accs_train.append(b_acc_train); accs_train.append(acc_train); f1s_train.append(f1_train)

    # append the results to the dataframe
    df_results.loc[len(df_results)] = ['Dummy_most_frequent', np.mean(b_accs_test), np.std(b_accs_test), np.mean(accs_test), np.std(accs_test), 
                                       np.mean(f1s_test), np.std(f1s_test), np.mean(b_accs_train), np.std(b_accs_train), 
                                       np.mean(accs_train), np.std(accs_train), np.mean(f1s_train), np.std(f1s_train), 
                                       np.mean(fitting_times), np.mean(prediction_times)]
    return df_results

df_results = inter_dummy(df, df_results)
df_results

def tt_split(df, random_state=42):
    X = df.drop(columns='Class')
    y = df['Class']
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=random_state)
    # make dataframes from the arrays
    X_train = pd.DataFrame(X_train, columns=X.columns)
    X_test = pd.DataFrame(X_test, columns=X.columns)
    y_train = y_train.to_numpy()
    y_test = y_test.to_numpy()
    return X_train, X_test, y_train, y_test

def impression(df):
    X_train, X_test, y_train, y_test = tt_split(df)
    # scale
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)

    # support vector machine default
    svc = SVC()
    fitting_time = time()
    svc.fit(X_train, y_train)
    fitting_time = time() - fitting_time
    y_pred = svc.predict(X_test)
    prediction_time = time() - fitting_time
    b_acc_test, acc_test, f1_test = scorer(y_test, y_pred)
    b_acc_train, acc_train, f1_train = scorer(y_train, svc.predict(X_train))
    df_results.loc[len(df_results)] = ['SVC_default', b_acc_test, None, acc_test, None, 
                                       f1_test, None, b_acc_train, None, acc_train, 
                                       None, f1_train, None, fitting_time, prediction_time] 

    # AdaBoost with DecisionTreeClassifier
    ada = AdaBoostClassifier(estimator=DecisionTreeClassifier())
    fitting_time = time()
    ada.fit(X_train, y_train)
    fitting_time = time() - fitting_time
    y_pred = ada.predict(X_test)
    prediction_time = time() - fitting_time
    b_acc_test, acc_test, f1_test = scorer(y_test, y_pred)
    b_acc_train, acc_train, f1_train = scorer(y_train, ada.predict(X_train))
    df_results.loc[len(df_results)] = ['AdaBoost_default', b_acc_test, None, acc_test, None, 
                                       f1_test, None, b_acc_train, None, acc_train, 
                                       None, f1_train, None, fitting_time, prediction_time]

display(df_results)

def opt_2d(df, param_grid, model):
    X_train, X_test, y_train, y_test = tt_split(df)

    # standart scaler
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)
    
    # Create a dictionary with the hyperparameters
    scores_test = np.zeros((len(list(param_grid.values())[0]), len(list(param_grid.values())[1])))
    scores_train = np.zeros((len(list(param_grid.values())[0]), len(list(param_grid.values())[1])))
    for i_C,C in enumerate(list(param_grid.values())[0]):
        for i_gamma, gamma in enumerate(list(param_grid.values())[1]): 
            params = {list(param_grid.keys())[0]: C, list(param_grid.keys())[1]: gamma} 
            model.set_params(**params)  
            # Create a SVC model
            model.fit(X_train, y_train)
            y_pred = model.predict(X_test)
            # test score
            balanced_accuracy = balanced_accuracy_score(y_test, y_pred)
            # i_C is the row -> y-axis, i_gamma is the column -> x-axis
            scores_test[i_C, i_gamma] = balanced_accuracy
            # train score
            y_pred = model.predict(X_train)
            balanced_accuracy = balanced_accuracy_score(y_train, y_pred)
            scores_train[i_C, i_gamma] = balanced_accuracy
    return scores_test, scores_train


def two_d_hist(param_grid, scores_train, scores_test, type_scale = 'log'):
    fig = make_subplots(rows=1, cols=2, subplot_titles=('Test', 'Train'))

    x_res = list(param_grid.values())[1]
    y_res = list(param_grid.values())[0]

    x_name = list(param_grid.keys())[1]
    y_name = list(param_grid.keys())[0]
    custom_colorscale = [
        [0, 'yellow'],
        [0.9, 'green'],
        [1, 'red']
    ]
    heatmap_train = go.Heatmap(
        z=scores_train,
        x=x_res,
        y=y_res,
        colorscale=custom_colorscale,
        showscale=True,
        text=[[f'{val:.4f}' for val in row] for row in scores_train],
        texttemplate="%{text}",
        textfont={"size":10}
    )
    fig.add_trace(heatmap_train, row=1, col=2)

    heatmap_test = go.Heatmap(
        z=scores_test,
        x=x_res,
        y=y_res,
        colorscale=custom_colorscale,
        showscale=False,
        text=[[f'{val:.4f}' for val in row] for row in scores_test],
        texttemplate="%{text}",
        textfont={"size":10}
    )
    fig.add_trace(heatmap_test, row=1, col=1)

    fig.update_layout(
        height=600,
        width=1000,
        title_text='Train and Test Heatmaps'
    )

    fig.update_xaxes(title_text=x_name, row=1, col=1, type=type_scale)
    fig.update_yaxes(title_text=y_name, row=1, col=1, type=type_scale)
    fig.update_xaxes(title_text=x_name, row=1, col=2, type=type_scale)
    fig.update_yaxes(title_text=y_name, row=1, col=2, type=type_scale)

    fig.show()

param_grid = {
        'C': [10e-3, 10e-2, 10e-1, 1, 10, 100, 1000, 10000, 100000] ,
        'gamma': [10e-6,10e-5, 10e-4, 10e-3, 10e-2, 10e-1, 1, 10, 100]
    }

model = SVC()

scores_test, scores_train = opt_2d(df,param_grid, model)
two_d_hist(param_grid, scores_train, scores_test)

def bal_accuracy(df, params):
    for i in range(20):
        # train test split
        X_train, X_test, y_train, y_test = tt_split(df, random_state=i)

        # standart scaler
        scaler = StandardScaler()
        X_train = scaler.fit_transform(X_train)
        X_test = scaler.transform(X_test)
        
        weights = [None,'balanced']
        bal_acc = [[],[]]
        for i_weights, weight in enumerate(weights):
            svc = SVC(**params, class_weight=weight)
            svc.fit(X_train, y_train)
            y_pred = svc.predict(X_test)
            b_acc = balanced_accuracy_score(y_test, y_pred)
            bal_acc[i_weights].append(b_acc)
    print(f'The balanced accuracy of the SVC model with class weight None is {np.mean(bal_acc[0]):.4f} and with class weight balanced is {np.mean(bal_acc[1]):.4f}')

params = {'C': 100, 'gamma': 0.01}
bal_accuracy(df, params)

The balanced accuracy of the SVC model with class weight None is 0.9394 and with class weight balanced is 0.9403

def ovo_ovr(df, params):
    # train test split
    X_train, X_test, y_train, y_test = tt_split(df)

    # standart scaler
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)

    shapes = ['ovo', 'ovr']
    fit_time = [[],[]]
    pred_time = [[],[]]
    for i_sh, shape in enumerate(shapes):
        for i in range(60):
            # Create a SVC model
            svc = SVC(**params, decision_function_shape=shape)
            a = time()
            svc.fit(X_train, y_train)
            b = time()
            y_pred = svc.predict(X_test)
            c = time()
            b_acc = balanced_accuracy_score(y_test, y_pred)
            fit_time[i_sh].append(b-a)
            pred_time[i_sh].append(c-b)   
        print(f'The balanced accuracy of the SVC model with decision function shape {shape} is {b_acc}')    
    fig, ax = plt.subplots()
    ax.plot(fit_time[0], label='ovo', color = 'blue')
    ax.plot(fit_time[1], label='ovr', color = 'orange')
    ax.set_title('Fitting times over itterations')
    ax.set_xlabel('Iteration')
    ax.set_ylabel('Time')
        # draw horizontal line for mean value
    plt.axhline(np.mean(fit_time[0]), color='blue', linestyle='--', label='mean ovo')
    plt.axhline(np.mean(fit_time[1]), color='orange', linestyle='--', label='mean ovr')
    ax.legend()
    plt.show()

params = {'C': 100, 'gamma': 0.01, 'class_weight':'balanced'}
ovo_ovr(df, params)

The balanced accuracy of the SVC model with decision function shape ovo is 0.9379958189788775
The balanced accuracy of the SVC model with decision function shape ovr is 0.9379958189788775

def scorer(y_true, y_pred):
    b_acc = balanced_accuracy_score(y_true, y_pred)
    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='weighted')
    return b_acc, acc, f1


def find_good_params(df=None, opt_par=[], opt_par_name= None, random_state=42, x_label='x_label', y_label='Balanced Accuracy', params = {}):
     # Train-test split
    X_train, X_test, y_train, y_test = tt_split(df, random_state=random_state)

    # Standard scaler
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)
    
    train_bal_acc = []
    test_bal_acc = []
    for i_opt, par in enumerate(opt_par):
        params[opt_par_name] = par
        model.set_params(**params)
        # Train the model
        model.fit(X_train, y_train)
        # Evaluate on test data
        y_pred_test = model.predict(X_test)
        b_acc_test, acc_test, f1_test = scorer(y_test, y_pred_test)
        test_bal_acc.append(b_acc_test)

        # Evaluate on train data
        y_pred_train = model.predict(X_train)
        b_acc_train, acc_train, f1_train = scorer(y_train, y_pred_train)
        train_bal_acc.append(b_acc_train)

    opt_par_array = np.array(opt_par)  # Convert opt_par to array

    # Create the plot with labels for both lines
    fig = go.Figure()
    fig.add_trace(go.Scatter(x=opt_par_array, y=train_bal_acc, mode='lines', name='Train Balanced Accuracy'))
    fig.add_trace(go.Scatter(x=opt_par_array, y=test_bal_acc, mode='lines', name='Test Balanced Accuracy'))

    fig.update_layout(title='Balanced Accuracy for different ' + x_label + ' values',
                      xaxis_title=x_label,
                      yaxis_title=y_label)

    fig.show()

params = {'C': 100, 'gamma': 0.01, 'kernel': 'poly', 'degree': 3, 'class_weight':'balanced'}
coef0 = np.linspace(0, 4, 25)
find_good_params(df = df, opt_par = coef0, opt_par_name='coef0', x_label='coef0',  params=params)

params = {'C': 100, 'gamma': 0.01, 'kernel': 'poly', 'degree': 3, 'class_weight':'balanced', 'coef0': 0.5}
tol = np.linspace(0.0001, 1, 20)
find_good_params(df = df, opt_par = tol, opt_par_name='tol', x_label='tol',  params=params, random_state=42)

# we will use the best parameters from the previous steps
params = {'C': 100, 'gamma': 0.01, 'coef0': 0.5, 'kernel':'poly', 'class_weight':'balanced'}
degree = range(1, 10)
find_good_params(df = df, opt_par = degree, opt_par_name='degree', x_label='degree',  params=params)

def nested_cross_validator(model_pipe = None, outer_cv = 3, inner_cv = 3, param_grid = {},estimator = None, x = None, y = None, n_repeats = 1, ev_scores = []) -> list:
    """
    Perform nested cross validation to find the best model.
    Also encoding of encoding and scaling can be treated as hyperparameters.
    """
    # import gridsearch and repeated stratified k fold
    from sklearn.model_selection import GridSearchCV, RepeatedStratifiedKFold
    from sklearn.metrics import make_scorer
    from tqdm import tqdm

    # create the outer loop
    bal_accuracys_train = []
    accuracys_train = []
    f1_scores_train = []
    bal_accuracys_test = []
    accuracys_test = []
    f1_scores_test = []
    outer_loop = RepeatedStratifiedKFold(n_splits=outer_cv, n_repeats=n_repeats, random_state = 1)

    for train_ix, test_ix in tqdm(outer_loop.split(x, y), total=outer_cv*n_repeats, desc='Progress'):
        # now gridsearch
        grid_search = GridSearchCV(estimator=model_pipe, param_grid=param_grid, n_jobs= 4, cv=inner_cv, scoring=make_scorer(estimator), verbose=1)
        # fit grid search
        search = grid_search.fit(x[train_ix], y[train_ix])
        # get the best model
        best_model = search.best_estimator_
        # predict the best model
        yhat = best_model.predict(x[test_ix])
        # evaluate the model
        b_acc_test, acc_test, f1_test = scorer(y[test_ix], yhat)
        # append the results
        bal_accuracys_test.append(b_acc_test); accuracys_test.append(acc_test); f1_scores_test.append(f1_test)
        # store the result
        yhat = best_model.predict(x[train_ix])
        b_acc_train, acc_train, f1_train = scorer(y[train_ix], yhat)
        # append the results
        bal_accuracys_train.append(b_acc_train); accuracys_train.append(acc_train); f1_scores_train.append(f1_train)
        # evaluate the model
        
    # store the results
    # add the results to the dataframe, using concatentation
    mean_fit_time = np.mean(search.cv_results_['mean_fit_time'])
    mean_score_time = np.mean(search.cv_results_['mean_score_time'])

    df_results.loc[len(df_results)] = ['SVM_n_cv', np.mean(bal_accuracys_test), np.std(bal_accuracys_test), np.mean(accuracys_test), np.std(accuracys_test),
                                   np.mean(f1_scores_test), np.std(f1_scores_test), np.mean(bal_accuracys_train), np.std(bal_accuracys_train), np.mean(accuracys_train), np.std(accuracys_train),
                                   np.mean(f1_scores_train), np.std(f1_scores_train), mean_fit_time, mean_score_time]
    # save results
    save_results(df_results)

def scorer(y_true, y_pred):
    b_acc = balanced_accuracy_score(y_true, y_pred)
    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='weighted')
    return b_acc, acc, f1

def save_results(results):
    results.to_csv(f'results/results_svm.csv', index=False)

svm_pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('svc', SVC(class_weight='balanced', decision_function_shape='ovr'))
])

param_grid = {
    'svc__gamma':[0.001, 0.01, 0.1],
    'svc__C': [1, 10, 100],
    'svc__coef0': [0.4,0.6,0.8],
    'svc__kernel': ['poly', 'rbf'],
    'svc__degree': [1, 2, 3, 4, 5],
}

outer_cv = 10
inner_cv = 10
n_repeats = 2

X = df.drop(columns='Class').values
y = df['Class'].values

'''Uncoment to run the nested cross validation'''
# nested_cross_validator(model_pipe = svm_pipe, outer_cv = outer_cv, inner_cv = inner_cv, param_grid = param_grid, estimator = balanced_accuracy_score, x = X, y = y, n_repeats = n_repeats)

# read in the results
results = pd.read_csv('results/results_svm.csv')
display(results)

# Ausing algorithm = 'SAMME' because it is deprecated
model = AdaBoostClassifier(estimator=DecisionTreeClassifier(), algorithm='SAMME')

   # Create a dictionary with the hyperparameters
param_grid = {
        'n_estimators': np.linspace(1, 100, 10, dtype=int),
        'estimator__ccp_alpha': np.arange(0,.002, 0.0002),
    }
'''Uncoment to run the 2d optimization'''

scores_test, scores_train = opt_2d(df, param_grid, model)
two_d_hist(param_grid, scores_train, scores_test, type_scale='linear')

#   ada = AdaBoostClassifier(n_estimators= 90, estimator=DecisionTreeClassifier(max_depth=depth, ccp_alpha=0.001, class_weight='balanced'))

max_depth = range(1, 30, 3)
opt_par_name = 'estimator__max_depth'
params = {'n_estimators': 90, 'estimator__ccp_alpha': 0.001, 'estimator__class_weight':'balanced'}

find_good_params(df = df, opt_par = max_depth, opt_par_name=opt_par_name, x_label='max_depth',  params=params)

def nested_cross_validator(model_pipe = None, outer_cv = 3, inner_cv = 3, param_grid = {},estimator = None, x = None, y = None, n_repeats = 1, ev_scores = []) -> list:
    """
    Perform nested cross validation to find the best model.
    Also encoding of encoding and scaling can be treated as hyperparameters.
    """
    # import gridsearch and repeated stratified k fold
    from sklearn.model_selection import GridSearchCV, RepeatedStratifiedKFold
    from sklearn.metrics import make_scorer
    from tqdm import tqdm

    # create the outer loop
    bal_accuracys_train = []
    accuracys_train = []
    f1_scores_train = []
    bal_accuracys_test = []
    accuracys_test = []
    f1_scores_test = []
    outer_loop = RepeatedStratifiedKFold(n_splits=outer_cv, n_repeats=n_repeats, random_state = 1)

    for train_ix, test_ix in tqdm(outer_loop.split(x, y), total=outer_cv*n_repeats, desc='Progress'):
        # now gridsearch
        grid_search = GridSearchCV(estimator=model_pipe, param_grid=param_grid, n_jobs= 4, cv=inner_cv, scoring=make_scorer(estimator), verbose=1)
        # fit grid search
        search = grid_search.fit(x[train_ix], y[train_ix])
        # get the best model
        best_model = search.best_estimator_
        # predict the best model
        yhat = best_model.predict(x[test_ix])
        # evaluate the model
        b_acc_test, acc_test, f1_test = scorer(y[test_ix], yhat)
        # append the results
        bal_accuracys_test.append(b_acc_test); accuracys_test.append(acc_test); f1_scores_test.append(f1_test)
        # store the result
        yhat = best_model.predict(x[train_ix])
        b_acc_train, acc_train, f1_train = scorer(y[train_ix], yhat)
        # append the results
        bal_accuracys_train.append(b_acc_train); accuracys_train.append(acc_train); f1_scores_train.append(f1_train)
        # evaluate the model
    # store the results
    # add the results to the dataframe, using concatentation
    mean_fit_time = np.mean(search.cv_results_['mean_fit_time'])
    mean_score_time = np.mean(search.cv_results_['mean_score_time'])

    df_results.loc[len(df_results)] = ['AdaBoost_Tree_Nest', np.mean(bal_accuracys_test), np.std(bal_accuracys_test), np.mean(accuracys_test), np.std(accuracys_test),
                                   np.mean(f1_scores_test), np.std(f1_scores_test), np.mean(bal_accuracys_train), np.std(bal_accuracys_train), np.mean(accuracys_train), np.std(accuracys_train),
                                   np.mean(f1_scores_train), np.std(f1_scores_train), mean_fit_time, mean_score_time]
    # model name
    save_results(df_results)


def scorer(y_true, y_pred):
    b_acc = balanced_accuracy_score(y_true, y_pred)
    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='weighted')
    return b_acc, acc, f1

def save_results(results):
    results.to_csv('results/results_adab_tree.csv', index=False)

# make the pipeline with AdaBoost using DecisionTreeClassifier, SAMME.R algorithm will be romoved in future versions, so we will use SAMME
ada_pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('ada', AdaBoostClassifier(estimator=DecisionTreeClassifier(class_weight='balanced'), algorithm='SAMME'))
])

# this will give a good result for the SVM
X = df.drop(columns='Class').values
y = df['Class'].values

param_grid = {
    'ada__estimator__ccp_alpha': [0.0004, 0.0006, 0.0008],
    'ada__n_estimators': [80,90,100],
}
'''Uncoment to run the nested cross validation'''
#result = nested_cross_validator(model_pipe = ada_pipe, outer_cv = 8, inner_cv = 8, param_grid = param_grid, estimator = balanced_accuracy_score, x = X, y = y, n_repeats = 2)

df_results = pd.read_csv('results/results_adab_tree.csv')
display(df_results)

import joblib

def final_model(df, params, opt_par):
    # Train-test split
    X_train, X_test, y_train, y_test = tt_split(df, random_state=42)

    # Standard scaler
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)
    
    # make dataframes
    X_train = pd.DataFrame(X_train, columns=df.drop(columns='Class').columns)
    X_test = pd.DataFrame(X_test, columns=df.drop(columns='Class').columns)


    # Create a Support Vector Machine model
    svc = SVC(**params)
    
    # GridSearchCV
    grid_search = GridSearchCV(estimator=svc, param_grid=opt_par, n_jobs=-1, cv=6, scoring=make_scorer(balanced_accuracy_score), verbose=0)
    search = grid_search.fit(X_train, y_train)
    
    # Get the best model
    best_model = search.best_estimator_
    
    # save the model
    joblib.dump(best_model, 'results/best_model.pkl')

def load_evaluate_model(df):
    model = joblib.load('results/best_model.pkl')
    # split with same random state
    X_train, X_test, y_train, y_test = tt_split(df, random_state=42)
    # Standard scaler
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)

    # make
    X_train = pd.DataFrame(X_train, columns=df.drop(columns='Class').columns)
    X_test = pd.DataFrame(X_test, columns=df.drop(columns='Class').columns)

    # Predict using the best model
    yhat_test = model.predict(X_test)
    yhat_train = model.predict(X_train)

    return model, yhat_test, y_test, yhat_train, y_train


params = {'class_weight': 'balanced'}

param_grid = {
    'gamma': [0.001, 0.01, 0.1],
    'C': [1, 10, 100],
    'coef0': [0, 0.1,0.2, 0.3],
    'kernel': ['poly', 'rbf'],
    'degree': [1, 2, 3],
}


final_model(df, params, param_grid)
best_model, predictions, y_test, predictions_train, y_train = load_evaluate_model(df)
best_model

SVC(C=10, class_weight='balanced', coef0=0, degree=1, gamma=0.1)

SVC(C=10, class_weight='balanced', coef0=0, degree=1, gamma=0.1)

def plot_bal_acc_class_plotly(y_true, y_pred, classes):
    """
    Plot the balanced accuracy for each class using Plotly.
    """
    # Calculate the balanced accuracy for each class
    class_accuracies = []
    for cls in classes:
        # Binary classification for the current class
        y_true_bin = (y_true == cls).astype(int)
        y_pred_bin = (y_pred == cls).astype(int)
        # Calculate balanced accuracy
        b_acc = balanced_accuracy_score(y_true_bin, y_pred_bin)
        class_accuracies.append(b_acc)
    
    # Create a DataFrame
    b_acc_df = pd.DataFrame(class_accuracies, index=classes, columns=['Balanced Accuracy'])
    b_acc_df.reset_index(inplace=True)
    b_acc_df.rename(columns={'index': 'Class'}, inplace=True)
    
    # Create the plot
    fig = px.bar(
        b_acc_df, 
        x='Balanced Accuracy', 
        y='Class', 
        title='Balanced Accuracy for Each Class',
        labels={'Balanced Accuracy': 'Balanced Accuracy', 'Class': 'Class'},
        color='Balanced Accuracy',
        color_continuous_scale='viridis',
        orientation='h',
        range_x=[0, 1]  # Balanced accuracy ranges from 0 to 1
    )
    
    # Add the accuracy values inside the bars
    fig.update_traces(text=b_acc_df['Balanced Accuracy'].round(4), textposition='inside')

    fig.update_layout(height=600)
    
    fig.show()

# Assuming y_test and predictions are defined, and df is your DataFrame
plot_bal_acc_class_plotly(y_test, predictions, df['Class'].unique())

import joblib
from sklearn.inspection import permutation_importance

def feature_importance(model, X, y):
    """
    Plot the feature importance using feature permutation with Plotly.
    """
    # Create the permutation importance object
    result = permutation_importance(model, X, y, n_repeats=20, random_state=42, n_jobs=-1, scoring='balanced_accuracy')
    
    # Extract importances and feature names
    importances = result['importances']
    feature_names = X.columns
    
    # Create a DataFrame for the importances
    importance_df = pd.DataFrame(importances.T, columns=feature_names)

    # rearange column with descendig mean order
    importance_df = importance_df[importance_df.mean().sort_values(ascending=True).index]

    
    fig = px.box(importance_df ,title='Permutation Feature Importance', 
           labels={'value': 'Decrease in balanced accuracy', 'variable': 'Feature'}, orientation='h', height=700)

    fig.show()

# Load the model
model = joblib.load('results/best_model.pkl')

# Split the data
X_train, X_test, y_train, y_test = tt_split(df)

# Scaling
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Convert back to DataFrames
X_train = pd.DataFrame(X_train, columns=df.drop(columns='Class').columns)
X_test = pd.DataFrame(X_test, columns=df.drop(columns='Class').columns)

# Calculate and plot feature importance
feature_importance(model, X_test, y_test)

Feature	Meaning
Area (A)	The area of a bean zone and the number of pixels within its boundaries.
Perimeter (P)	Bean circumference defined as the length of its border.
Major axis length (L)	The distance between the ends of the longest line that can be drawn from a bean.
Minor axis length (l)	The longest line that can be drawn from the bean while standing perpendicular to the main axis.
Aspect ratio (K)	Defines the relationship between the major axis length (L) and minor axis length (l).
Eccentricity (Ec)	Eccentricity of the ellipse having the same moments as the region.
Convex area (C)	Number of pixels in the smallest convex polygon that can contain the area of a bean seed.
Equivalent diameter (Ed)	The diameter of a circle having the same area as a bean seed area.
Extent (Ex)	The ratio of the pixels in the bounding box to the bean area.
Solidity (S)	Also known as convexity. The ratio of the pixels in the convex shell to those found in beans.
Roundness (R)	Calculated with the following formula: (4πA)/(P^2)
Compactness (CO)	Measures the roundness of an object: Equivalent diameter (Ed) divided by major axis length (L).
ShapeFactor1 (SF1)	Defined as the ratio of the major axis length (L) to the area (A): $$\frac{L}{A}$$
ShapeFactor2 (SF2)	Defined as the ratio of the minor axis length (l) to the area (A): $$\frac{l}{A}$$
ShapeFactor3 (SF3)	Defined as the ratio of the area (A) to the product of half the major axis length (L/2) and half the minor axis length (l/2) times π: $$\frac{A}{\left(\frac{L}{2} \cdot \frac{l}{2} \cdot \pi\right)}$$
ShapeFactor4 (SF4)	Defined as the ratio of the area (A) to the product of half the minor axis length (l/2) and half the major axis length (L/2) times π: $$\frac{A}{\left(\frac{l}{2} \cdot \frac{L}{2} \cdot \pi\right)}$$
Class	The type of dry bean: Seker, Barbunya, Bombay, Cali, Dermosan, Horoz, and Sira.

	Area	Perimeter	MajorAxisLength	MinorAxisLength	AspectRation	Eccentricity	ConvexArea	EquivDiameter	Extent	Solidity	roundness	Compactness	ShapeFactor1	ShapeFactor2	ShapeFactor3	ShapeFactor4	Class
0	28395	610.291	208.178117	173.888747	1.197191	0.549812	28715	190.141097	0.763923	0.988856	0.958027	0.913358	0.007332	0.003147	0.834222	0.998724	SEKER
1	28734	638.018	200.524796	182.734419	1.097356	0.411785	29172	191.272750	0.783968	0.984986	0.887034	0.953861	0.006979	0.003564	0.909851	0.998430	SEKER
2	29380	624.110	212.826130	175.931143	1.209713	0.562727	29690	193.410904	0.778113	0.989559	0.947849	0.908774	0.007244	0.003048	0.825871	0.999066	SEKER
3	30008	645.884	210.557999	182.516516	1.153638	0.498616	30724	195.467062	0.782681	0.976696	0.903936	0.928329	0.007017	0.003215	0.861794	0.994199	SEKER
4	30140	620.134	201.847882	190.279279	1.060798	0.333680	30417	195.896503	0.773098	0.990893	0.984877	0.970516	0.006697	0.003665	0.941900	0.999166	SEKER

	Area	Perimeter	MajorAxisLength	MinorAxisLength	AspectRation	Eccentricity	ConvexArea	EquivDiameter	Extent	Solidity	roundness	Compactness	ShapeFactor1	ShapeFactor2	ShapeFactor3	ShapeFactor4
min	20420.000000	524.736000	183.601165	122.512653	1.024868	0.218951	20684.000000	161.243764	0.555315	0.919246	0.489618	0.640577	0.002778	0.000564	0.410339	0.947687
max	254616.000000	1985.370000	738.860153	460.198497	2.430306	0.911423	263261.000000	569.374358	0.866195	0.994677	0.990685	0.987303	0.010451	0.003665	0.974767	0.999733
mean	53048.284549	855.283459	320.141867	202.270714	1.583242	0.750895	53768.200206	253.064220	0.749733	0.987143	0.873282	0.799864	0.006564	0.001716	0.643590	0.995063
std	29324.095717	214.289696	85.694186	44.970091	0.246678	0.092002	29774.915817	59.177120	0.049086	0.004660	0.059520	0.061713	0.001128	0.000596	0.098996	0.004366

Setting	Value
Outer cross validations	10
Inner cross validations	10
Repeat nested cross validation	2
Gridsearch on same outer split	No

Setting	Value
Outer cross validations	8
Inner cross validations	8
Gridsearch	1
Gridsearch on same outer split	No

Hyperparameter	Values
C	[1, 10, 100]
gamma	[0.001, 0.01, 0.1]
coef0	[0.4, 0.6, 0.8]
degree	[1, 2, 3, 4, 5]
svc__kernel	['poly', 'rbf']

Hyperparameter	Values
ada__estimator__ccp_alpha	[0.0004, 0.0006, 0.0008]
ada__n_estimators	[80, 90, 100]

	Model	Mean Balanced Accuracy Test	Std Balanced Accuracy Test	Mean Accuracy Test	Std Accuracy Test	Mean F1 Score Test	Std F1 Score Test	Mean Balanced Accuracy Train	Std Balanced Accuracy Train	Mean Accuracy Train	Std Accuracy Train	Mean F1 Score Train	Std F1 Score Train	Fit Time	Prediction Time
0	SVM_GridSearchCV	0.934958	0.007651	0.921204	0.008305	0.921456	0.008247	0.936352	0.00152	0.922701	0.002029	0.922957	0.001991	1.057793	0.199260
1	AdaBoost_Tree_Nest	0.935744	0.008180	0.925024	0.007641	0.925020	0.007615	1.000000	0.00000	1.000000	0.000000	1.000000	0.000000	37.880772	0.061533

Metric	SVM_GridSearchCV	AdaBoost_Tree_Nest
Test Performance
Mean Balanced Accuracy	0.9350	0.9357
Std Balanced Accuracy	0.0077	0.0082
Mean Accuracy	0.9212	0.9250
Std Accuracy	0.0083	0.0076
Mean F1 Score	0.9215	0.9250
Std F1 Score	0.0082	0.0076
Train Performance
Mean Balanced Accuracy	0.9364	1.0000
Std Balanced Accuracy	0.0015	0.0000
Mean Accuracy	0.9227	1.0000
Std Accuracy	0.0020	0.0000
Mean F1 Score	0.9230	1.0000
Std F1 Score	0.0020	0.0000
Timing
Mean Fit Time (seconds)	1.0578	37.8808
Mean Prediction Time (seconds)	0.1993	0.0615

	Duplicate Rows
Class
HOROZ	68

Portfolio Examination - Machine Learning -Tom Ruge¶

Story - Task 1¶

The Data - Task 2¶

Side Information¶

Detailed Description¶

Physical Implementation¶

Initial Data Analysis (IDA) - Task 3¶

Exploratory Data Analysis (EDA) - Task 4¶

Original Feautures¶

Feature engineered Features¶

A First Impression - Task 5¶

Metrics¶

Baseline¶

Selection of 2 Classifiers¶

AdaBoost (Adaptive Boosting)¶

How AdaBoost Works¶

Support Vector Machines (SVMs)¶

How SVM Works¶

AdaBoost and SVM with Default Settings¶

Nested Cross Validation - Task 6¶

How Nested Cross-Validation Works¶

Support Vector Machine¶

Choosing a Reasonable Hyperparameter Grid¶

Nested Cross Validation¶

AdaBoost¶

Choosing a Reasonable Hyperparameter Grid¶

Nested Cross Validation¶

Analysis of the Final Model - Task 7¶

Summary of Results¶

Performance¶

Overfitting Risk¶

Training and Prediction Time¶

Analysis of the Individual Classifications¶

Feature Importance - Task 8¶

Basic Idea¶

Analysis of Permutation Feature Importance Plot¶

Key Observations:¶

Implications for Model Improvement:¶

Conclusion:¶

Conclusion and Future Work¶

Summary¶

Usabilty of Applied Methodology¶

Usability for the Company¶

Limitations and Pitfalls¶

Ideas and Future Work¶